package edu.unc.ils.memai.rw;

import java.io.BufferedReader;




import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import maui.stemmers.PorterStemmer;
import maui.stemmers.Stemmer;
import maui.util.Candidate;
import maui.util.Counter;

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;

import com.aliasi.io.FileExtensionFilter;
import com.aliasi.util.Files;

import edu.unc.ils.memai.extract.CandidateExtractor;
import edu.unc.ils.memai.extract.MauiCandidateExtractor;
import edu.unc.ils.memai.extract.NASACandidateExtractor;
import edu.unc.ils.memai.tokenize.MauiTokenizer;
import edu.unc.ils.memai.tokenize.Tokenizer;
import edu.unc.ils.memai.vocab.H2Vocabulary;
import edu.unc.ils.memai.vocab.LuceneVocabulary;
import edu.unc.ils.memai.vocab.MauiVocabulary;
import edu.unc.ils.memai.vocab.Vocabulary;
import edu.unc.ils.util.ValueComparator;

public class RandomWalkWithPhraseExtractor 
{
    Vocabulary vocabulary = null;
    Stemmer stemmer = null;
    CandidateExtractor extractor = null;
    static int vocabSize = 0;
    
    static int NUM_LEVELS = 11;
    
    
    public static void main(String[] args) throws Exception 
    {
        // Get the options specified
        CommandLineParser parser = new BasicParser( );

        Options options = getOptions();
        CommandLine commandLine = parser.parse( options, args );

        if (commandLine.hasOption("h"))
        {
            // Print the help message
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp( "java " + RandomWalkWithPhraseExtractor.class.getCanonicalName(), options );
        }
        else
        {
            String dirPath = commandLine.getOptionValue("d");
            String vocabularyPath = commandLine.getOptionValue("v");
            
            String vocabModel = commandLine.getOptionValue("vm");
            if (vocabModel == null)
                vocabModel = "lucene";
            
            String candidateModel = commandLine.getOptionValue("cm");
            if (candidateModel == null)
                candidateModel = "maui";
            
            boolean doPrint = commandLine.hasOption("p");
            boolean doStem = commandLine.hasOption("s");
            boolean doVary = commandLine.hasOption("a");
            boolean doIgnoreCounts = commandLine.hasOption("i");
            String optN = commandLine.getOptionValue("n");
            String optK = commandLine.getOptionValue("k");
            String optBtWeight = commandLine.getOptionValue("bt");
            String optNtWeight = commandLine.getOptionValue("nt");
            String optRtWeight = commandLine.getOptionValue("rt");
            String optMinPhrase = commandLine.getOptionValue("mp");
            
            int n = 500;
            int k = 3;
            int btWeight = 1;
            int ntWeight = 1;
            int rtWeight = 1;
            int minPhraseOccur = 1;
            
            if (optN != null)
                n = Integer.valueOf(optN);
            
            if (optK != null)
                k = Integer.valueOf(optK);
            
            if (optBtWeight != null)
                btWeight = Integer.valueOf(optBtWeight);
            
            if (optNtWeight != null)
                ntWeight = Integer.valueOf(optNtWeight);
            
            if (optRtWeight != null)
                rtWeight = Integer.valueOf(optRtWeight);
            
            if (optMinPhrase != null)
                minPhraseOccur = Integer.valueOf(optMinPhrase);
            
            Vocabulary v = new LuceneVocabulary();
            if (vocabModel != null)
            {
                if (vocabModel.equals("maui"))
                    v = new MauiVocabulary();
                else if (vocabModel.equals("h2"))
                    v = new H2Vocabulary();
            }

            Stemmer stemmer = new PorterStemmer();
            v.setStemmer(stemmer);
            
            List<String> stopwords = FileUtils.readLines(
                    new File("src/main/resources/stopwords_en.txt"), "UTF-8");
            
            v.setStopwords(stopwords);
            v.open(vocabularyPath);
            v.setThreshold(10);
            v.setLowerCase(true);
            //v.load("/Users/cwillis/dev/hive/hive-data/nalt/naltStoreEn");
            v.load("/Users/cwillis/dev/hive/hive-data/agrovoc/agrovocStore");
            vocabSize = v.getSize();
            
            CandidateExtractor ce = new MauiCandidateExtractor();
            if (candidateModel != null)
            {
                if (candidateModel.equals("nasa"))
                    ce = new NASACandidateExtractor();
            }
            ce.setStemmer(stemmer);
            ce.setVocabulary(v);
            ce.setMaxPhraseLength(3);
            ce.setMinPhraseLength(1);
            ce.setMinOccurFrequency(minPhraseOccur);
            ce.setStopwords(stopwords);
            
            RandomWalkWithPhraseExtractor rw = new RandomWalkWithPhraseExtractor();
            rw.setCandidateExtractor(ce);
            rw.setStemmer(stemmer);
            rw.setVocabulary(v);
            
            
            int[] Ns = new int[] { 10, 100, 200, 400 } ;
            int[] Ks = new int[] { 0, 4, 8, 12, 16, 20 } ;
            
                
            if (dirPath != null)
            {
                File dir = new File(dirPath);
                File[] files = dir.listFiles(new FileExtensionFilter(".txt"));

                matchOnly(rw, v, files);
                
                if (!doVary)
                    pass(rw, v, files, n, k, btWeight, ntWeight, rtWeight, doStem, doPrint, doIgnoreCounts);
                else
                {
                    List<String> weights = FileUtils.readLines(new File("src/main/resources/weights"), "UTF-8");
                    
                    for (int N: Ns)
                    {
                        for (int K: Ks)
                        {
                            for (String weight: weights)
                            {
                                String[] w = weight.split(" ");
                                int bt = Integer.valueOf(w[0]);
                                int nt = Integer.valueOf(w[1]);
                                int rt = Integer.valueOf(w[2]);
                                pass(rw, v, files, N, K, bt, nt, rt, doStem, doPrint, doIgnoreCounts);
                            }
                        }
                    }
                }
            }
        }
    }
    
    public static void matchOnly (RandomWalkWithPhraseExtractor rw, Vocabulary v, 
            File[] files) throws Exception
    {
        System.out.println("match only ");
        DescriptiveStatistics[] precision = new DescriptiveStatistics[NUM_LEVELS];
        DescriptiveStatistics[] recall = new DescriptiveStatistics[NUM_LEVELS];
        
        for (int i=0; i<NUM_LEVELS; i++)
        {
            precision[i] = new DescriptiveStatistics();
            recall[i] = new DescriptiveStatistics();
        }
        
        for (File file: files)
        {
            if (!file.isFile()) 
                continue;
            
            String text = Files.readFromFile(file, "UTF-8");
            String fileName = file.getName();
            String fileId = fileName.substring(0, fileName.lastIndexOf("."));
            
            List<String> headings = readHeadings(file.getParent() + File.separator + fileId + ".subjects");
            
            Map<String, Candidate> matches = rw.getCandidates(text);
            Map<Integer, Integer> termCountMap = new HashMap<Integer, Integer>();
            for (Candidate match: matches.values())
            {
                Map<String, Counter> instances = match.getFullForms();
                
                for (String orig: instances.keySet())
                {
                    Counter c = instances.get(orig);
                    
                    for (int i=0; i < c.value(); i++)
                    {
                        String phrase = match.getTitle();
                        if (phrase == null)
                            continue;
                        int conceptId = v.getConceptId(phrase);

                        Integer count = termCountMap.get(conceptId);
                        if (count == null)
                            count = 1;
                        else 
                            count++;
                            
                        termCountMap.put(conceptId, count);

                    }
                }
            }
            
            ValueComparator vc = new ValueComparator(termCountMap);
            Map<Integer, Integer> sortedMap = new TreeMap<Integer, Integer>(vc);
            sortedMap.putAll(termCountMap);
            List<double[]> result = rw.countMatches( sortedMap, headings); 
            
            int numRelevant = headings.size();
            //int numRetrieved = sortedMap.size();
            
            int vocabSize = v.getSize();
            int numFound = 0;
            if (result.size() > 0)
                numFound = (int)result.get(result.size() - 1)[2];

            // Calculate at each rank
            precision[0].addValue(1);
            recall[0].addValue(0);
            
            double prevp = 1;
            double prevr = 0;
            int lasti = 0;
            int lastj = 0;
            for (int i=1; i<NUM_LEVELS; i++)
            {
                double fixedr = (double)i/10;
                for (int j=lastj; j < result.size(); j++)
                {
                    double[] pr = result.get(j);
                    double p = pr[0];
                    double r = pr[1];
                    if ( r > fixedr)
                    {
                        double pinterp = lerp(prevr, prevp, r, p, fixedr);
                        precision[i].addValue(pinterp);
                        recall[i].addValue(fixedr);
                        lasti = i;
                        lastj = j;
                        break;
                    }
                    prevp = p;
                    prevr = r;
                }
            }
            
            int k = 0;
            int open = NUM_LEVELS - (lasti+2);
            int missing = numRelevant - numFound;
            double remaining = (vocabSize - matches.size());
            if (open > 0)
                remaining = ((vocabSize - matches.size())/open);
            for (int i=lasti+1; i<NUM_LEVELS; i++)
            {
                double fixedr = (double)i/10;
                
                double p = 0;
                if (k==0 || k == (NUM_LEVELS - 1))
                {
                    p = (missing / remaining)/2;
                }
                else
                {
                    p = (missing / remaining);
                }
                precision[i].addValue(p);
                recall[i].addValue(fixedr);
                prevp = p;
                prevr = fixedr;
                
                k++;
            }
            
        }

        double meanPrecision[] = new double[NUM_LEVELS];
        double meanRecall[] = new double[NUM_LEVELS];
        
        for (int i=0; i<NUM_LEVELS; i++)
        {
            meanPrecision[i] = precision[i].getMean();
            meanRecall[i] = recall[i].getMean();
            System.out.println(meanRecall[i] + "," + meanPrecision[i]);
        }
    }

    public static void pass(RandomWalkWithPhraseExtractor rw, Vocabulary v, File[] files,   
            int n, int k, int btWeight, int ntWeight, int rtWeight, boolean doStem, 
            boolean doPrint, boolean doIgnoreCounts) throws Exception
    {
        System.out.println("n=" + n + ", k=" + k + ", bt=" + btWeight + ", nt=" + ntWeight + ", rt=" + rtWeight);
        DescriptiveStatistics[] precision = new DescriptiveStatistics[NUM_LEVELS];
        DescriptiveStatistics[] recall = new DescriptiveStatistics[NUM_LEVELS];

        for (int i=0; i<NUM_LEVELS; i++)
        {
            precision[i] = new DescriptiveStatistics();
            recall[i] = new DescriptiveStatistics();
        }
        
        //System.out.println("Reading files");
        for (File file: files)
        {
            if (!file.isFile()) 
                continue;
            
            String text = Files.readFromFile(file, "UTF-8");
            String fileName = file.getName();
            String fileId = fileName.substring(0, fileName.lastIndexOf("."));
            
            List<String> headings = readHeadings(file.getParent() + File.separator + fileId + ".subjects");
  
            Map<String, Candidate> matches = rw.getCandidates(text);
  
            Map<Integer, Integer> sortedMap = rw.doWalk(fileId, text, headings, matches, n, k, 
                    btWeight, ntWeight, rtWeight, doStem, doPrint, doIgnoreCounts);
            
            List<double[]> result = rw.countMatches( sortedMap, headings); 
            
            int numRelevant = headings.size();
            //int numRetrieved = sortedMap.size();
            
            int vocabSize = v.getSize();
            int numFound = 0;
            if (result.size() > 0)
                numFound = (int)result.get(result.size() - 1)[2];

            // Calculate at each rank
            precision[0].addValue(1);
            recall[0].addValue(0);
            
            double prevp = 1;
            double prevr = 0;
            int lasti = 0;
            for (int i=1; i<NUM_LEVELS; i++)
            {
                double fixedr = (double)i/10;
                boolean found = false;
                for (int j=0; j < result.size(); j++)
                {
                    double[] pr = result.get(j);
                    double p = pr[0];
                    double r = pr[1];
                    if ( r > fixedr)
                    {
                        double pinterp = lerp(prevr, prevp, r, p, fixedr);
                        precision[i].addValue(pinterp);
                        recall[i].addValue(fixedr);
                        found = true;
                        lasti = i;
                        break;
                    }
                    prevp = p;
                    prevr = r;
                }
                /*
                if (!found)
                {
                    double p = (double)(numRelevant - numFound)/vocabSize;
                    //double pinterp = lerp(lastr, lastp, 1, minPrecision, fixedr);
                    precision[i].addValue(p);
                    recall[i].addValue(fixedr);
                    prevp = p;
                    prevr = fixedr;
                }
                */
                
            }
            
            int kk = 0;
            int open = NUM_LEVELS - (lasti+2);
            int missing = numRelevant - numFound;
            double remaining = (vocabSize - matches.size());
            if (open > 0)
                remaining = ((vocabSize - matches.size())/open);
            for (int i=lasti+1; i<NUM_LEVELS; i++)
            {
                double fixedr = (double)i/10;
                
                double p = 0;
                if (kk==0 || kk == (NUM_LEVELS - 1))
                {
                    p = (missing / remaining)/2;
                }
                else
                {
                    p = (missing / remaining);
                }
                precision[i].addValue(p);
                recall[i].addValue(fixedr);
                prevp = p;
                prevr = fixedr;
                
                kk++;
            }
            
            /*
            double numRelevant = result.getNumRelevant();
            double[] numFound = result.getNumFound();
            double[] numRelevantFound = result.getNumRelevantFound();
            
            // Calculate at each rank
            for (int i=0; i < NUM_LEVELS; i++)
            {
                // p is the number of correct results 
                //   divided by the number of all returned results
                double p = 0;
                if (numFound[i] > 0)
                    p = numRelevantFound[i] / numFound[i];
                
                // r is the number of correct results 
                //   divided by the number of results that should have been returned
                double r = 0;
                if (numRelevant > 0)            
                    r = numRelevantFound[i] / numRelevant;
                
                precision[i].addValue(p);
                recall[i].addValue(r);
            }
            */
        }

        double meanPrecision[] = new double[NUM_LEVELS];
        double meanRecall[] = new double[NUM_LEVELS];
        
        for (int i=0; i<NUM_LEVELS; i++)
        {
            meanPrecision[i] = precision[i].getMean();
            meanRecall[i] = recall[i].getMean();
            System.out.println(meanRecall[i] + "," + meanPrecision[i]);
        }
        
       
        /*
        for (double j=0; j < 1.0; j+=.1)
        {
            double p = 0.0;
            for (int i = 0; i<meanRecall.length; i++)
            {
                double r = meanRecall[i];
                if (r >= j && r < j+.1)
                {
                    p = meanPrecision[i];
                    break;
                }
            }  
            System.out.println(df.format(j) + "," + df.format(p));
        }
        */
    }

    public static double lerp(double x0, double y0, double x1, double y1, double x)
    {   
        double y = y0 + (x - x0) * ( (y1 - y0)/(x1 - x0) );
        return y;
    }
    /**
     * Returns the CLI options
     * @return
     */
    public static Options getOptions()
    {
        Options options = new Options();
        
        Option dirPath = new Option("d", true, "Path to input directory");
        dirPath.setRequired(false);
        options.addOption(dirPath);
        
        Option vocabularyPath = new Option("v", true, "Path to vocabulary store");
        vocabularyPath.setRequired(false);
        options.addOption(vocabularyPath);
        
        Option vocabModel = new Option("vm", true, "h2, lucene, maui");
        vocabModel.setRequired(false);
        options.addOption(vocabModel);
        
        Option candidateModel = new Option("cm", true, "lucene, maui, nasa");
        candidateModel.setRequired(false);
        options.addOption(candidateModel);
        
        options.addOption("h", false, "Print this help message");
        options.addOption("p", false, "Print headings");
        options.addOption("s", false, "Use stemmer");
        options.addOption("n", true, "Number of walks. Defaults to 500.");
        options.addOption("k", true, "Node length. Defaults to 3");
        options.addOption("bt", true, "Broader term weight. Defaults to 1");
        options.addOption("nt", true, "Narrower term weight. Defaults to 1");
        options.addOption("rt", true, "Related term weight. Defaults to 1");
        options.addOption("mp", true, "Minimum phrase occurrence");
        options.addOption("a", false, "Run through variations");
        options.addOption("i", false, "Skips term counts");
       
        return options;
    }

    
    public void setCandidateExtractor(CandidateExtractor extractor)
    {
        this.extractor = extractor;
    }
    
    public void setVocabulary(Vocabulary vocabulary)
    {
        this.vocabulary = vocabulary;
    }
    
    public void setStemmer(Stemmer stemmer)
    {
        this.stemmer = stemmer;
    }
    
    public Map<String, Candidate> getCandidates(String text) throws Exception
    {
        Tokenizer tok = new MauiTokenizer();
        List<String> segments = tok.tokenize(text, false);
        
        Map<String, Candidate> matches = extractor.getCandidates(segments);
        return matches;
    }
    
    public Map<Integer, Integer>  doWalk(String name, String text, List<String> headings, 
            Map<String, Candidate> matches, int N, int K, int btWeight, 
            int ntWeight, int rtWeight, boolean stem, boolean print, boolean ignoreCounts) 
        throws Exception
    {
        Map<Integer, Integer> termCountMap = new HashMap<Integer, Integer>();
        Map<String, Integer> origTermCountmap = new HashMap<String, Integer>(); 
        
        //FileWriter fw1 = new FileWriter("output/" + name + "-" + N + ".matches");
        for (Candidate match: matches.values())
        {
            Map<String, Counter> instances = match.getFullForms();
            
            for (String orig: instances.keySet())
            {
                int cnts = 1;
                if (!ignoreCounts)
                {
                    Counter c = instances.get(orig);
                    cnts = c.value();
                }
                
                for (int i=0; i < cnts; i++)
                {
                    String phrase = match.getTitle();
                    if (phrase == null)
                        continue;
                    int conceptId = vocabulary.getConceptId(phrase);
                    Integer c2 = origTermCountmap.get(orig);
                    if (c2 == null)
                        c2 = 1;
                    else 
                        c2++;
                    origTermCountmap.put(orig, c2);
                    
                    for (int n=0; n<N; n++)
                    {
                        
                        int lastConceptId = conceptId;
                        for (int k=0; k<K; k++)
                        {
                            int nextConceptId = vocabulary.getNextRandom(lastConceptId, btWeight, ntWeight, rtWeight);
                            lastConceptId = nextConceptId;
                        }
                        
                        String label = vocabulary.getPrefLabel(lastConceptId);
                        Integer count = termCountMap.get(lastConceptId);
                        if (count == null)
                            count = 1;
                        else 
                            count++;
                        
                        termCountMap.put(lastConceptId, count);
                        
                        //fw1.write(match.getBestFullForm() + "," + phrase + "," + label + "\r\n");
                    }
                }
            }
        }
        //fw1.close();
        
        ValueComparator vc = new ValueComparator(termCountMap);
        Map<Integer, Integer> sortedMap = new TreeMap<Integer, Integer>(vc);
        sortedMap.putAll(termCountMap);
        
        //int numRelevant = headings.size();



       // FileWriter fw = new FileWriter("output/" + name + "-" + N  + ".out");
       // for (Integer id: sortedMap.keySet()) 
       // {
       //     String term = vocabulary.getPrefLabel(id);
       //     Integer count = sortedMap.get(id);
       //     fw.write(term + "\t" + count + "\r\n");
       // }
       // fw.close();
        
        
       // FileWriter fw3 = new FileWriter("output/" + name + "-" + N + ".count");
       // for (String term: origTermCountmap.keySet()) 
       // {
       //     Integer count = origTermCountmap.get(term);
       //     fw3.write(term + "\t" + count + "\r\n");
       // }
       // fw3.close();

        //Result result = new Result();
        //result.setNumFound(numFound);
        //result.setNumRelevant(numRelevant);
        //result.setNumRelevantFound(numRelevantFound);
        //result.setNumMatches(matches.size());
        
        return sortedMap;
        
    }
    
    
    
    public List<double[]> countMatches(Map<Integer, Integer> sortedIdCountMap, List<String> headings) 
        throws Exception
    {
        List<double[]> result = new ArrayList<double[]>();
        
        int numRelevant = headings.size();
        
        int numFound = 1;
        int numRelevantFound = 0;
        for (Integer id: sortedIdCountMap.keySet()) 
        {
            String term = vocabulary.getPrefLabel(id).toLowerCase();
            Integer count = sortedIdCountMap.get(id);
            if (count != null) 
            {
                if (count > 0 && headings.contains(term)) {     
                    numRelevantFound++;
                    double p  = (double)numRelevantFound/(double)numFound;
                    double r  = (double)numRelevantFound/(double)numRelevant;
                    double f = (double)numRelevantFound;
                    result.add(new double[] {p, r, f});
                }
            }
            numFound++;
        }
        return result;
    }
     
    public static String readText(String path) throws Exception
    {
        String text = "";
        BufferedReader br = new BufferedReader(new FileReader(path));
        String line;
        while ((line = br.readLine()) != null)
            text += line + " ";
 
        return text;
    }
    
    public static List<String> readHeadings(String subjectPath) throws Exception
    {
        List<String> headings = new ArrayList<String>();
        BufferedReader br = new BufferedReader(new FileReader(subjectPath));
        String line;
        while ((line = br.readLine()) != null)
            headings.add(line.toLowerCase());
 
        return headings;
    }
    
    static double fmeasure(double beta, double p, double r)
    {
        //double f = 2 * (p/r);           

        double betaSq = Math.pow(beta, 1);
        double f = 0;
        if (p > 0 && r > 0)
           f = (1 + betaSq) * ( (p*r) / ((betaSq * p) + r) );
        
        return f;
    }
    
    static double losee(double numFoundAndGoodType, double numGoodType, double numFoundAndBadType, double numBadType)
    {
        double num = 0;
        if (numFoundAndGoodType > 0 && numGoodType > 0)
            num = numFoundAndGoodType/numGoodType;

        double den = 0;
        if (numFoundAndBadType > 0 && numBadType > 0)
            den = numFoundAndBadType/numBadType;
        
        double log2 = 0;
        if (num > 0 && den > 0)
        {
            double l = num/den;
        
            // Log base 2
            log2 = Math.log(l)/Math.log(2);
        }
        return log2;
    }
    
    static List<int[]> readWeights() throws Exception
    {
        List<int[]> weights = new ArrayList<int[]>();
        BufferedReader br = new BufferedReader(new FileReader("src/main/resources/weights"));
        String line;
        while ((line = br.readLine()) != null)
        {
            if (line.startsWith("#"))
                continue;
            String[] fields = line.split(" ");
            int[] w = new int[fields.length];
            for (int i=0; i<fields.length; i++)
            {
                w[i] = Integer.valueOf(fields[i]);
            }
            weights.add(w);
        }
 
        return weights;
    }
}
